package com.luych.crowler;

import edu.uci.ics.crawler4j.crawler.Page;
import edu.uci.ics.crawler4j.crawler.WebCrawler;
import edu.uci.ics.crawler4j.url.WebURL;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.*;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.UUID;
import java.util.regex.Pattern;

public class Crawler extends WebCrawler{

    private final Logger logger = LoggerFactory.getLogger(this.getClass());

    private final static Pattern FILTERS = Pattern.compile(".*(\\.(jpg|png|jpeg|bmp|gif|mp4|wmv|avi|rmvb|3gp|flv))$");
    //private final static Pattern FILTERS = Pattern.compile(".*(\\.(mp4|wmv|avi|rmvb|3gp|flv))$");

    @Override
    public boolean shouldVisit(Page referringPage, WebURL url) {
        String urlString = url.getURL();
        logger.info(urlString);
        if (FILTERS.matcher(urlString).matches()) {
            String extName = urlString.substring(urlString.lastIndexOf(".") + 1);
            String fileName = UUID.randomUUID().toString();
            String finalName = downLoadFromUrl(urlString, fileName + "." + extName, "D:\\crawl");
            logger.info(finalName + "--->>" + urlString);
        }
        return true;
    }

    @Override
    public void visit(Page page) {


    }

    private String downLoadFromUrl(String urlStr, String fileName, String savePath) {
        try {

            URL url = new URL(urlStr);
            HttpURLConnection conn = (HttpURLConnection) url.openConnection();
            // 设置超时间为3秒
            conn.setConnectTimeout(3 * 1000);
            // 防止屏蔽程序抓取而返回403错误
            conn.setRequestProperty("User-Agent", "Mozilla/4.0 (compatible; MSIE 5.0; Windows NT; DigExt)");

            // 得到输入流
            InputStream inputStream = conn.getInputStream();
            // 获取自己数组
            byte[] getData = readInputStream(inputStream);

            // 文件保存位置
            File saveDir = new File(savePath);
            if (!saveDir.exists()) {
                saveDir.mkdir();
            }
            File file = new File(saveDir + File.separator + fileName);
            FileOutputStream fos = new FileOutputStream(file);
            fos.write(getData);
            if (fos != null) {
                fos.close();
            }
            if (inputStream != null) {
                inputStream.close();
            }
            return saveDir + File.separator + fileName;
        } catch (Exception e) {
            e.printStackTrace();
        }
        return "";

    }

    private byte[] readInputStream(InputStream inputStream) throws IOException {
        byte[] buffer = new byte[1024];
        int len;
        ByteArrayOutputStream bos = new ByteArrayOutputStream();
        while ((len = inputStream.read(buffer)) != -1) {
            bos.write(buffer, 0, len);
        }
        bos.close();
        return bos.toByteArray();
    }
}
